require 'rubygems'
require 'hpricot'
require 'net/http'

module Mkuz

    class Downloader
        
        $USER_AGENT = 'iTunes/10.3.1 (Windows; Microsoft Windows 7 Ultimate Edition Service Pack 1 (Build 7601)) AppleWebKit/533.21.1'
        
        def initialize(dir)
            @dir = dir
        end
        
        
        def run(link, title)
            path = "#{@dir}/#{title}_#{Time.now.localtime.strftime("%Y-%m-%d")}.csv"
            
            puts "downloading from #{link} to #{path} started"
            
            list = parse_list(link)
            store_list(list, path)
            
            puts "downloading finished"
        end
        
        
        def parse_list(data_link)
            puts "openning #{data_link}..."

            parsed_items = []
            
            data_uri = URI.parse(data_link)
            puts "data uri: #{data_uri.host}:#{data_uri.port}#{data_uri.path}?#{data_uri.query}"

            Net::HTTP.start( data_uri.host, data_uri.port ) do |http|
                doc = Hpricot(http.get("#{data_uri.path}?#{data_uri.query}", "User-Agent" => $USER_AGENT ).body)

                puts "ok"

                num_of_pages = doc.at("//div[@total-number-of-pages]")['total-number-of-pages'].to_i
                puts "num_of_pages=#{num_of_pages}"
            
                (1..num_of_pages).each do |page_index|
                    page_uri = URI.parse('http://itunes.apple.com/WebObjects/MZStore.woa/wa/roomPageContentMatrixFragment')
                    page_uri.query = "#{data_uri.query.split('&')[0]}&page=#{page_index}&sort=2"
                    puts "page uri: #{page_uri.host}:#{page_uri.port}#{page_uri.path}?#{page_uri.query}"
                    
                    page = Hpricot(http.get("#{page_uri.path}?#{page_uri.query}", "User-Agent" => $USER_AGENT ).body)

                    page.search("//div[@aria-label]").each do |item|
                        puts "extracing app #{item['aria-label']}"
                
                        item_link = item.at("//ul[@class='list']/li[@class='name']/a")['href']
                        
                        parsed_item = parse_item(http, item_link)
                        parsed_items << parsed_item
                        #break; #TODO remove
                    end
                end
            end
            
            return parsed_items
        end
        
        def parse_item(http, link)
            puts "parsing app #{link} started"
            
            parsed_item = {}
            
            item_uri = URI.parse(link)
            raw_item = Hpricot(http.get("#{item_uri.path}?#{item_uri.query}", "User-Agent" => $USER_AGENT ).body)
            
            name = raw_item.at("//div[@class='title']/h1/a").inner_text
            id    = raw_item.at("//div/button[@adam-id]")['adam-id'].to_i
            release_date = raw_item.at("//ul[@class='list']/li[@class='release-date']").inner_text.split(':')[1].strip
            #puts "release date = #{release_date}"
            
            size = ""
            raw_item.search("//ul[@class='list']/li").each do |li|
                if (li.inner_text.split(':')[0] == 'Size') then
                    size = li.inner_text.split(':')[1].strip
                end
            end
            
            rated = raw_item.at("//div[@class='app-rating']/a").inner_text
            
            ratings_link = 'http://itunes.apple.com/WebObjects/MZStore.woa/wa/customerReviews?displayable-kind=11&id=' + id.to_s
            ratings_uri = URI.parse(ratings_link)
            
            raw_ratings = Hpricot(http.get("#{ratings_uri.path}?#{ratings_uri.query}", "User-Agent" => $USER_AGENT ).body)
            ratings = { "5 stars" => 0, "4 stars"=> 0, "3 stars"=> 0, "2 stars"=> 0, "1 star"=> 0}
            
            ratings_title = raw_ratings.at("//div[@class='ratings-histogram']/a")
            if (!ratings_title.nil? && ratings_title.inner_text == 'Average rating for the current version: ') then
                puts 'Enough Votes!!'
                raw_ratings.search("//div[@class='vote']").each do |stars|
                    stars_key = stars.at("//div[@class='stars']")['aria-label']
                    stars_val = stars.at("//span[@class='total']").inner_text.to_i
                    ratings[stars_key] = stars_val
                end
                puts "ratings=#{ratings.inspect}"
            end
            
            raw_desc = raw_item.at("div[@class='product-info']/div/p")
            raw_desc.search("br").each do |br| br.swap("\n") end

            full_desc = raw_desc.inner_text.gsub("\t", "    ");
            short_desc = ""
            full_desc.each { |line| short_desc=line.chomp; break; }


            
            parsed_item['name'] = name
            parsed_item['id'] = id
            parsed_item['link'] = link
            parsed_item['release_date'] = release_date
            parsed_item['size'] = size
            parsed_item['rated'] = rated
            parsed_item.merge!(ratings)
            
            parsed_item["short_desc"] = short_desc
            parsed_item["short_desc_length"] = short_desc.length
            parsed_item["full_desc"] = full_desc.gsub("\n", ". ")
            parsed_item["full_desc_length"] = full_desc.length
            
            puts "ok"
            
            return parsed_item
        end
        
        def store_list(list, path)
            header = ""
            content = ""
            list.each do |item|
                item.keys.each {|k| header += "#{k}\t" } if header == ""
                content += "\n"
                item.values.each { |v| content += "#{v.to_s}\t" }
            end
            
            puts "writing file to #{path}..."
            File.open(path, 'w') { |f| f.write(header + content) }
            puts "ok"
        end
        
        protected :parse_list, :parse_item, :store_list
    end

end #module
